<!DOCTYPE html>
<html lang="zh-CN">
  <head>
    
<meta charset="UTF-8"/>
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1"/>


<meta http-equiv="Cache-Control" content="no-transform" />
<meta http-equiv="Cache-Control" content="no-siteapp" />

<meta name="theme-color" content="#f8f5ec" />
<meta name="msapplication-navbutton-color" content="#f8f5ec">
<meta name="apple-mobile-web-app-capable" content="yes">
<meta name="apple-mobile-web-app-status-bar-style" content="#f8f5ec">



  <meta name="description" content="Python爬虫,收藏功能实现记录"/>




  <meta name="keywords" content="开发, flask, python, lxml, 八一" />



  <meta name="baidu-site-verification" content="HhUstaSjr0" />



  <meta name="google-site-verification" content="UA-102975942-1" />






  <link rel="alternate" href="/atom.xml" title="八一">




  <link rel="shortcut icon" type="image/x-icon" href="/favicon.ico?v=2.6.0" />



<link rel="canonical" href="https://bay1.top/2017/10/08/矿大新闻热点爬虫系统搬砖记录/"/>


<link rel="stylesheet" type="text/css" href="/css/style.css?v=2.6.0" />
<link rel="stylesheet" type="text/css" href="/css/prettify.css" media="screen" />
<link rel="stylesheet" type="text/css" href="/css/sons-of-obsidian.css" media="screen" />



  <link rel="stylesheet" type="text/css" href="/lib/fancybox/jquery.fancybox.css" />




  
  <script id="baidu_analytics">
    var _hmt = _hmt || [];
    (function() {
      var hm = document.createElement("script");
      hm.src = "https://hm.baidu.com/hm.js?9a885cc9fb6cd7bcef579deb8efe8a70";
      var s = document.getElementsByTagName("script")[0];
      s.parentNode.insertBefore(hm, s);
    })();
  </script>



  <script id="google_analytics">
    (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
        (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
        m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
        })(window,document,'script','//www.google-analytics.com/analytics.js','ga');

        ga('create', 'UA-102975942-1', 'auto');
        ga('send', 'pageview');
  </script>










    <title> Python爬虫,收藏功能实现记录 - 八一 </title>
  </head>

  <body><div id="mobile-navbar" class="mobile-navbar">
  <div class="mobile-header-logo">
    <a href="/." class="logo">八一</a>
  </div>
  <div class="mobile-navbar-icon">
    <span></span>
    <span></span>
    <span></span>
  </div>
</div>

<nav id="mobile-menu" class="mobile-menu slideout-menu">
  <ul class="mobile-menu-list">
    
      <a href="/archives">
        <li class="mobile-menu-item">
          
          
            文章
          
        </li>
      </a>
    
      <a href="/tags">
        <li class="mobile-menu-item">
          
          
            标签
          
        </li>
      </a>
    
      <a href="/about">
        <li class="mobile-menu-item">
          
          
            关于/友链
          
        </li>
      </a>
    
      <a href="/search">
        <li class="mobile-menu-item">
          
          
            站内搜索
          
        </li>
      </a>
    
  </ul>
</nav>

    <div class="container" id="mobile-panel">
      <header id="header" class="header"><div class="logo-wrapper">
  <a href="/." class="logo">八一</a>
</div>

<nav class="site-navbar">
  
    <ul id="menu" class="menu">
      
        <li class="menu-item">
          <a class="menu-item-link" href="/archives">
            
            
              文章
            
          </a>
        </li>
      
        <li class="menu-item">
          <a class="menu-item-link" href="/tags">
            
            
              标签
            
          </a>
        </li>
      
        <li class="menu-item">
          <a class="menu-item-link" href="/about">
            
            
              关于/友链
            
          </a>
        </li>
      
        <li class="menu-item">
          <a class="menu-item-link" href="/search">
            
            
              站内搜索
            
          </a>
        </li>
      
    </ul>
  
</nav>

      </header>

      <main id="main" class="main">
        <div class="content-wrapper">
          <div id="content" class="content">
            
  
  <article class="post">
    <header class="post-header">
      <h1 class="post-title">
        
          Python爬虫,收藏功能实现记录
        
      </h1>

      <div class="post-meta">
        <span class="post-time">
          2017-10-08
        </span>
        
        
        
      </div>
    </header>

    
    
  <div class="post-toc" id="post-toc">
    <h2 class="post-toc-title">文章目录</h2>
    <div class="post-toc-content">
      <ol class="toc"><li class="toc-item toc-level-2"><a class="toc-link" href="#爬虫部分"><span class="toc-text">爬虫部分</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#大众网页爬取"><span class="toc-text">大众网页爬取</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#JS翻页的网站"><span class="toc-text">JS翻页的网站</span></a></li></ol></li><li class="toc-item toc-level-2"><a class="toc-link" href="#储存数据"><span class="toc-text">储存数据</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#图表的构造"><span class="toc-text">图表的构造</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#收藏功能的实现"><span class="toc-text">收藏功能的实现</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#form搜索功能"><span class="toc-text">form搜索功能</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#一些小知识点"><span class="toc-text">一些小知识点</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#bootstrap-wtf样式自定"><span class="toc-text">bootstrap/wtf样式自定</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#Bootstrap栅格系统是好东西"><span class="toc-text">Bootstrap栅格系统是好东西</span></a></li></ol></li><li class="toc-item toc-level-2"><a class="toc-link" href="#小结"><span class="toc-text">小结</span></a></li></ol>
    </div>
  </div>


    <div class="post-content">
      
        <p>经历了20天的时间,慌慌张张的把这个准备参加比赛的Demo做了出来 <a id="more"></a><br><a href="https://github.com/bayi27/SpiderCumtNews" target="_blank" rel="noopener">源代码</a><br>看着比赛官网的往届作品,en…………<br>由于比较Low,失败告终<br>这里就简单记录一下一些遇到的问题</p>
<blockquote>
<p>本项目以我们学校学院网作为基点,通过Python爬取主要学院网站学院新闻,通知公告,学生.学术动态三类数据<br>集成了一个校园Web信息热点分析与推荐系统<br>本系统为学校网站管理员生成各类信息报表,同时本系统面向学生群体集中数据并展示</p>
</blockquote>
<h2 id="爬虫部分"><a href="#爬虫部分" class="headerlink" title="爬虫部分"></a>爬虫部分</h2><blockquote>
<p>这个系统是围绕学院网站进行的,学校网站还是比较好爬取的<br>这里采取了requests和解析效率很高的lxml库爬取数据<br>lxml虽然没有Beautifulsoup闻名,但是速度更为优秀,操作更为便捷<br>同时爬虫这件事针对不同的网站采取的爬取规则也不一样,所以对于不同的网站要不同区别</p>
</blockquote>
<h3 id="大众网页爬取"><a href="#大众网页爬取" class="headerlink" title="大众网页爬取"></a>大众网页爬取</h3><blockquote>
<p>学院网站大部分都是采取相同的网站模板<br>因此也为数据的过滤提供了便利</p>
</blockquote>
<p>采取的思路为:新闻list链接–list首页读取尾页–生成list_dict–在list读取文章链接–文章链接里读取数据</p>
<p>这里任意抽取一个为例</p>
<figure class="highlight perl"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br></pre></td><td class="code"><pre><span class="line">url_get=requests.get(url,timeout=<span class="number">10</span>)</span><br><span class="line">collegenews_get_result=etree.HTML(collegenews_get.text) <span class="comment"># 转化为xpth能识别</span></span><br><span class="line">article_url_paths=result.xpath(<span class="string">"//div[@class='wk_new_lb_title']/a[@title]/@href"</span>)</span><br><span class="line">total_page=re.search(r<span class="string">'[0-9]+(?=[^0-9]*$)'</span>,str(end_page))</span><br><span class="line">...</span><br><span class="line"><span class="keyword">for</span> i in range(<span class="number">0</span>,<span class="keyword">int</span>(total_page)+<span class="number">1</span>):</span><br><span class="line">	new_link=link.replace(<span class="string">"list"</span>,<span class="string">"list"</span>+str(i))</span><br><span class="line">	link_list.append(new_link)</span><br><span class="line">...</span><br><span class="line">article_url_paths=result.xpath(<span class="string">"//tr/td/a[@class]/@href"</span>)</span><br><span class="line">...</span><br><span class="line"><span class="keyword">for</span> article_url_path in article_url_paths:</span><br><span class="line">	<span class="keyword">if</span> <span class="string">'.htm'</span> in article_url_path <span class="keyword">and</span> <span class="string">'http://'</span> <span class="keyword">not</span> in article_url_path:</span><br><span class="line">		article_url.append(<span class="string">'http://'</span>+str(article_url_root)+str(article_url_path))</span><br><span class="line">...</span><br><span class="line">article_url_get_result=etree.HTML(article_url_get.text) <span class="comment"># 转化为xpth能识别</span></span><br><span class="line">article_time=article_url_get_result.xpath(<span class="string">"//div[@class='article_other_info']/span[@id='Label_time']/text()"</span>)</span><br><span class="line">article_author=article_url_get_result.xpath(<span class="string">"//div[@class='article_other_info']/span[@class='arti_update']/text()"</span>)</span><br><span class="line">title=article_url_get_result.xpath(<span class="string">"//head/title/text()"</span>)</span><br><span class="line">clicknumber_url=article_url_get_result.xpath(<span class="string">"//table[@class='border2']/tr/td[@align='center']/span/@url"</span>)</span><br></pre></td></tr></table></figure>
<blockquote>
<p>这里的变量名称定义的很清楚了,提取出后的list可能是含有”杂质”的<br>这里可以使用正则匹配过滤一下<br>下面是我用到的几个,作用就是名字意思</p>
</blockquote>
<figure class="highlight markdown"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">date_re = re.compile(r'([<span class="string">0-9</span>]&#123;3&#125;[<span class="string">1-9</span>]|[<span class="string">0-9</span>]&#123;2&#125;[<span class="string">1-9</span>][<span class="symbol">0-9</span>]&#123;1&#125;|[<span class="string">0-9</span>]&#123;1&#125;[<span class="string">1-9</span>][<span class="symbol">0-9</span>]&#123;2&#125;|[<span class="string">1-9</span>][<span class="symbol">0-9</span>]&#123;3&#125;)-(((0[<span class="string">13578</span>]|1[<span class="string">02</span>])-(0[<span class="string">1-9</span>]|[<span class="string">12</span>][<span class="symbol">0-9</span>]|3[<span class="string">01</span>]))|((0[<span class="string">469</span>]|11)-(0[<span class="string">1-9</span>]|[<span class="string">12</span>][<span class="symbol">0-9</span>]|30))|(02-(0[<span class="string">1-9</span>]|[<span class="string">1</span>][<span class="symbol">0-9</span>]|2[0-8])))')</span><br><span class="line">author<span class="emphasis">_re=re.compile(r'[A-Za-z0-9_</span>\-\u4e00-\u9fa5]+')</span><br><span class="line">url_reg =re.match(r'^http?:\/\/([a-z0-9\-\.]+)[\/\?]?',link)</span><br></pre></td></tr></table></figure>
<h3 id="JS翻页的网站"><a href="#JS翻页的网站" class="headerlink" title="JS翻页的网站"></a>JS翻页的网站</h3><blockquote>
<p>针对这类的网站,我们可以采取抓包的方式分析数据交互<br>比如我碰到的这个网站,在page源码有这样一段代码</p>
</blockquote>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre></td><td class="code"><pre><span class="line">$.ajax(&#123;</span><br><span class="line">	type: <span class="string">"POST"</span>,</span><br><span class="line">    url: <span class="string">"/Services/ContentServices.asmx/GetContentList"</span>,</span><br><span class="line">    data: &#123;</span><br><span class="line">        pageIndex: start_index + <span class="number">1</span></span><br><span class="line">        , countPerPage: items_on_page</span><br><span class="line">        , category: category</span><br><span class="line">    &#125;,</span><br><span class="line">    success: function(data) &#123;&#125;</span><br><span class="line">    &#125;);</span><br></pre></td></tr></table></figure>
<blockquote>
<p>很明显通过post方式获取数据,尝试一个post之后发现获取的是json形式的数据<br>这时便可以构造代码了</p>
</blockquote>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br></pre></td><td class="code"><pre><span class="line">url=<span class="string">'http://xxxxx/Services/ContentServices.asmx/GetContentList'</span></span><br><span class="line">college_name=check_url(url)</span><br><span class="line">college = Colleges.query.filter_by(name=college_name).first()</span><br><span class="line">category=re.search(<span class="string">r'[0-9]+(?=[^0-9]*$)'</span>,str(link))</span><br><span class="line">headers=&#123;</span><br><span class="line">	<span class="string">'Content-Length'</span>: <span class="string">'41'</span>,</span><br><span class="line">	<span class="string">'Accept'</span>: <span class="string">'*/*'</span>,</span><br><span class="line">	<span class="string">'Origin'</span>: <span class="string">'http://xxxxx'</span>,</span><br><span class="line">	<span class="string">'X-Requested-With'</span>: <span class="string">'XMLHttpRequest'</span>,</span><br><span class="line">	<span class="string">'User-Agent'</span>: <span class="string">'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.90 Safari/537.36'</span>,</span><br><span class="line">	<span class="string">'Content-Type'</span>: <span class="string">'application/x-www-form-urlencoded'</span>,</span><br><span class="line">	<span class="string">'Referer'</span>: <span class="string">'http://xxxxx/default.aspx'</span>,</span><br><span class="line">	<span class="string">'Accept-Encoding'</span>: <span class="string">'gzip, deflate'</span>,</span><br><span class="line">	<span class="string">'Accept-Language'</span>: <span class="string">'zh-CN,zh;q=0.8'</span>,</span><br><span class="line">	<span class="string">'Connection'</span>: <span class="string">'close'</span></span><br><span class="line">&#125;</span><br><span class="line"><span class="keyword">for</span> pageIndex <span class="keyword">in</span> range(<span class="number">1</span>,<span class="number">30</span>):</span><br><span class="line">	data=&#123;</span><br><span class="line">		<span class="string">'pageIndex'</span>: pageIndex,</span><br><span class="line">		<span class="string">'countPerPage'</span>:<span class="string">'30'</span>,</span><br><span class="line">		<span class="string">'category'</span>: category[<span class="number">0</span>]</span><br><span class="line">	&#125;</span><br><span class="line">post_result=requests.post(url,headers=headers,data=data).content</span><br><span class="line">result=json.loads(etree.HTML(post_result).xpath(<span class="string">"//string/text()"</span>)[<span class="number">0</span>])[<span class="string">"Rows"</span>]</span><br><span class="line">...</span><br><span class="line">article_time=result[i][<span class="string">"添加时间"</span>][<span class="number">0</span>:<span class="number">10</span>]</span><br><span class="line">id=result[i][<span class="string">"内容ID"</span>]</span><br><span class="line">title=[result[i][<span class="string">"标题"</span>]]</span><br><span class="line">article_author=result[i][<span class="string">"姓名"</span>]</span><br><span class="line"><span class="keyword">if</span> result[i][<span class="string">"链接"</span>]==<span class="string">"True"</span>:</span><br><span class="line">	article_url=result[i][<span class="string">"链接地址"</span>]</span><br><span class="line"><span class="keyword">else</span>:</span><br><span class="line">	article_url=<span class="string">"http://sm.cumt.edu.cn/detail.aspx?m="</span>+category[<span class="number">0</span>]+<span class="string">"&amp;cid="</span>+id</span><br><span class="line">clicknumber=result[i][<span class="string">"浏览次数"</span>]</span><br></pre></td></tr></table></figure>
<h2 id="储存数据"><a href="#储存数据" class="headerlink" title="储存数据"></a>储存数据</h2><blockquote>
<p>在数据库设计时<br>数据库设计时,针对收藏功能,将用户和新闻分类采取多对多的模型设计<br>针对学院和新闻分类采取了一对多的模型设计</p>
</blockquote>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment"># 关联表</span></span><br><span class="line">UserToCollegeNews = db.Table(<span class="string">'UserToCollegeNews'</span>,</span><br><span class="line">            db.Column(<span class="string">'user_id'</span>, db.Integer, db.ForeignKey(<span class="string">'users.id'</span>), primary_key=<span class="keyword">True</span>),</span><br><span class="line">            db.Column(<span class="string">'collegenews_id'</span>, db.Integer, db.ForeignKey(<span class="string">'collegenews.id'</span>), primary_key=<span class="keyword">True</span>)</span><br><span class="line">)</span><br><span class="line"></span><br><span class="line">UserToCollegeNotices = db.Table(<span class="string">'UserToCollegeNotices'</span>,</span><br><span class="line">            db.Column(<span class="string">'user_id'</span>, db.Integer, db.ForeignKey(<span class="string">'users.id'</span>), primary_key=<span class="keyword">True</span>),</span><br><span class="line">            db.Column(<span class="string">'collegenotices_id'</span>, db.Integer, db.ForeignKey(<span class="string">'collegenotices.id'</span>), primary_key=<span class="keyword">True</span>)</span><br><span class="line">)</span><br><span class="line"></span><br><span class="line">UserToStudentWork = db.Table(<span class="string">'UserToStudentWork'</span>,</span><br><span class="line">            db.Column(<span class="string">'user_id'</span>, db.Integer, db.ForeignKey(<span class="string">'users.id'</span>), primary_key=<span class="keyword">True</span>),</span><br><span class="line">            db.Column(<span class="string">'studentworks_id'</span>, db.Integer, db.ForeignKey(<span class="string">'studentworks.id'</span>), primary_key=<span class="keyword">True</span>)</span><br><span class="line">)</span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">User</span><span class="params">(db.Model, UserMixin)</span>:</span></span><br><span class="line">    __tablename__ = <span class="string">'users'</span></span><br><span class="line">    ...</span><br><span class="line">    collegenews= db.relationship(<span class="string">'CollegeNews'</span>, secondary=UserToCollegeNews, backref=db.backref(<span class="string">'users'</span>, lazy=<span class="string">'dynamic'</span>))</span><br><span class="line">    collegenotices= db.relationship(<span class="string">'CollegeNotices'</span>, secondary=UserToCollegeNotices, backref=db.backref(<span class="string">'users'</span>, lazy=<span class="string">'dynamic'</span>))</span><br><span class="line">    studentworks= db.relationship(<span class="string">'StudentWork'</span>, secondary=UserToStudentWork, backref=db.backref(<span class="string">'users'</span>, lazy=<span class="string">'dynamic'</span>))</span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">Colleges</span><span class="params">(db.Model)</span>:</span></span><br><span class="line">	__tablename__=<span class="string">'colleges'</span></span><br><span class="line">	...</span><br><span class="line">	collegenew = db.relationship(<span class="string">'CollegeNews'</span>, backref=<span class="string">'colleges'</span>, lazy=<span class="string">'dynamic'</span>)</span><br><span class="line">	collegenotice = db.relationship(<span class="string">'CollegeNotices'</span>, backref=<span class="string">'colleges'</span>, lazy=<span class="string">'dynamic'</span>)</span><br><span class="line">	studentwork = db.relationship(<span class="string">'StudentWork'</span>, backref=<span class="string">'colleges'</span>, lazy=<span class="string">'dynamic'</span>)</span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">CollegeNews</span><span class="params">(db.Model)</span>:</span></span><br><span class="line">	__tablename__=<span class="string">'collegenews'</span></span><br><span class="line">	...</span><br><span class="line">	college = db.Column(db.Integer, db.ForeignKey(<span class="string">'colleges.name'</span>))</span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">CollegeNotices</span><span class="params">(db.Model)</span>:</span></span><br><span class="line">	__tablename__=<span class="string">'collegenotices'</span></span><br><span class="line">	...</span><br><span class="line">	college = db.Column(db.Integer, db.ForeignKey(<span class="string">'colleges.name'</span>))</span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">StudentWork</span><span class="params">(db.Model)</span>:</span></span><br><span class="line">	__tablename__=<span class="string">'studentworks'</span></span><br><span class="line">	...</span><br><span class="line">	college = db.Column(db.Integer, db.ForeignKey(<span class="string">'colleges.name'</span>))</span><br></pre></td></tr></table></figure>
<h2 id="图表的构造"><a href="#图表的构造" class="headerlink" title="图表的构造"></a>图表的构造</h2><blockquote>
<p>本系统使用Plotly引入各类的图表样式,根据官网的示例可以得知图标坐标轴读取的是list<br>所以我采用了Api的方式传输数据,在前端使用Ajax接收处理数据组合list生成图表<br>类似下面的形式,其他形式图表同理</p>
</blockquote>
<figure class="highlight js"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br></pre></td><td class="code"><pre><span class="line">$.ajax(&#123;</span><br><span class="line">    url: <span class="string">"../api/clicksum"</span>,</span><br><span class="line">    dataType: <span class="string">"json"</span>,</span><br><span class="line">    json: <span class="string">"callback"</span>,</span><br><span class="line">    success: <span class="function"><span class="keyword">function</span>(<span class="params">datas</span>) </span>&#123;</span><br><span class="line">        <span class="keyword">var</span> name_list = <span class="keyword">new</span> <span class="built_in">Array</span>();</span><br><span class="line">        <span class="keyword">var</span> clicksum_list = <span class="keyword">new</span> <span class="built_in">Array</span>();</span><br><span class="line">        <span class="keyword">for</span> (<span class="keyword">var</span> i = <span class="number">0</span>; i &lt;= datas.length - <span class="number">1</span>; i++) &#123;</span><br><span class="line">            name_list.push(datas[i][<span class="string">'name'</span>]);</span><br><span class="line">            clicksum_list.push(datas[i][<span class="string">'clicksum'</span>]);</span><br><span class="line">        &#125;</span><br><span class="line">        <span class="keyword">var</span> trace = &#123;</span><br><span class="line">            x: name_list,</span><br><span class="line">            y: clicksum_list,</span><br><span class="line">            type: <span class="string">'lines'</span>,</span><br><span class="line">        &#125;;</span><br><span class="line">        <span class="keyword">var</span> data_two = [trace];</span><br><span class="line">        <span class="keyword">var</span> layout_two = &#123;</span><br><span class="line">            width: <span class="string">'250px'</span>,</span><br><span class="line">        &#125;;</span><br><span class="line">        Plotly.newPlot(<span class="string">'my-graph-2'</span>, data_two, layout_two);</span><br><span class="line">    &#125;</span><br><span class="line">&#125;);</span><br></pre></td></tr></table></figure>
<h2 id="收藏功能的实现"><a href="#收藏功能的实现" class="headerlink" title="收藏功能的实现"></a>收藏功能的实现</h2><blockquote>
<p>这个功能的实现我也是头一次做<br>这里我实现的思路是:用户点击–跳转数据处理链接–数据处理函数Return原点击页面<br>这个问题我也询问了老组长,他给的思路是Ajax,类似上边提到的JS翻页的方式,使用api实现<br>只详细说我的思路吧</p>
</blockquote>
<p>首先在main蓝图里定义好路由,为了方便回跳到原来的链接,传入了多个参数<br>2333333333这里总感觉参数很多可控,很危险</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta">@main.route('/collect/&lt;is_collect&gt;/&lt;page&gt;/&lt;collegename&gt;/&lt;typename&gt;/&lt;collectarticle&gt;/&lt;title&gt;')</span></span><br><span class="line"><span class="meta">@login_required</span></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">article_collect</span><span class="params">(page,collectarticle,collegename,typename,title,is_collect)</span>:</span></span><br><span class="line">    user=User.query.filter(User.username==current_user.username).first()</span><br><span class="line">    query = CollegeNews.query.filter(CollegeNews.title==title).first()</span><br><span class="line">    <span class="keyword">if</span> is_collect==<span class="string">'1'</span>:</span><br><span class="line">    	user.collegenews.remove(query)</span><br><span class="line">    <span class="keyword">else</span>:</span><br><span class="line">        user.collegenews.append(query)</span><br><span class="line">    db.session.commit()</span><br><span class="line">    <span class="keyword">return</span> redirect(url_for(<span class="string">'.articles'</span>,page=page,collegename=collegename,typename=typename))</span><br></pre></td></tr></table></figure>
<blockquote>
<p>这里也加入了一个is_collect来判断是取消收藏还是收藏<br>下面是html部分代码</p>
</blockquote>
<figure class="highlight django"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre></td><td class="code"><pre><span class="line"><span class="xml"></span><span class="template-tag">&#123;% <span class="name"><span class="name">if</span></span> article <span class="keyword">in</span> is_collect %&#125;</span><span class="xml"></span></span><br><span class="line"><span class="xml">    <span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"</span></span></span><span class="template-variable">&#123;&#123; url_for('.article_collect',page=pagination.page,collegename=collegename,typename=typename,collectarticle=article,title=article.title,is_collect=1) &#125;&#125;</span><span class="xml"><span class="tag"><span class="string">"</span>&gt;</span></span></span><br><span class="line"><span class="xml">    <span class="tag">&lt;<span class="name">button</span> <span class="attr">class</span>=<span class="string">"btn btn-danger btn-xs"</span> <span class="attr">title</span>=<span class="string">'点击取消收藏'</span>&gt;</span></span></span><br><span class="line"><span class="xml"></span><span class="template-tag">&#123;% <span class="name"><span class="name">else</span></span> %&#125;</span><span class="xml"></span></span><br><span class="line"><span class="xml">    <span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"</span></span></span><span class="template-variable">&#123;&#123; url_for('.article_collect',page=pagination.page,collegename=collegename,typename=typename,collectarticle=article,title=article.title,is_collect=0) &#125;&#125;</span><span class="xml"><span class="tag"><span class="string">"</span>&gt;</span></span></span><br><span class="line"><span class="xml">    <span class="tag">&lt;<span class="name">button</span> <span class="attr">class</span>=<span class="string">"btn btn-default btn-xs"</span> <span class="attr">title</span>=<span class="string">'点击收藏'</span>&gt;</span></span></span><br><span class="line"><span class="xml"></span><span class="template-tag">&#123;% <span class="name"><span class="name">endif</span></span> %&#125;</span><span class="xml"></span></span><br><span class="line"><span class="xml">    <span class="tag">&lt;<span class="name">span</span> <span class="attr">class</span>=<span class="string">"glyphicon glyphicon-heart-empty"</span> <span class="attr">aria-hidden</span>=<span class="string">"true"</span>&gt;</span><span class="tag">&lt;/<span class="name">span</span>&gt;</span></span></span><br><span class="line"><span class="xml">    <span class="tag">&lt;/<span class="name">button</span>&gt;</span></span></span><br><span class="line"><span class="xml">    <span class="tag">&lt;/<span class="name">a</span>&gt;</span></span></span><br></pre></td></tr></table></figure>
<h2 id="form搜索功能"><a href="#form搜索功能" class="headerlink" title="form搜索功能"></a>form搜索功能</h2><blockquote>
<p>这个问题正好是前几天回答一个群友的问题,就是<a href="https://bayi87.github.io/2017/09/22/%E5%A6%82%E4%BD%95%E5%9C%A8flask%E5%AE%9E%E7%8E%B0form%E6%90%9C%E7%B4%A2/" target="_blank" rel="noopener">上一篇文章</a></p>
</blockquote>
<p><img src="https://s1.ax1x.com/2018/01/01/pSfcmn.jpg" alt="有趣"></p>
<h2 id="一些小知识点"><a href="#一些小知识点" class="headerlink" title="一些小知识点"></a>一些小知识点</h2><h3 id="bootstrap-wtf样式自定"><a href="#bootstrap-wtf样式自定" class="headerlink" title="bootstrap/wtf样式自定"></a>bootstrap/wtf样式自定</h3><blockquote>
<p>这是定义两个form为内联式,其他的可以具体搜索</p>
</blockquote>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">&#123; wtf.quick_form(form,form_type=&apos;inline&apos;) &#125;&#125;</span><br></pre></td></tr></table></figure>
<p>或者你不用wtf.quick_form,自己自定义action以及各种form样式</p>
<h3 id="Bootstrap栅格系统是好东西"><a href="#Bootstrap栅格系统是好东西" class="headerlink" title="Bootstrap栅格系统是好东西"></a>Bootstrap栅格系统是好东西</h3><blockquote>
<p>Bootstrap提供了一套响应式、移动设备优先的流式栅格系统，随着屏幕或视口（viewport）尺寸的增加，系统会自动分为最多12列<br>使用单一的一组 .col-md-<em> 栅格类,就可以创建一个基本的栅格系统,所有“列（column）必须放在 ” .row<br>使用 .col-md-offset-</em> 类可以将列向右侧偏移<br>使用 .col-sm-<em> 类针对平板设备<br>使用.col-xs-</em> 和 .col-md-*类针对超小屏幕和中等屏幕设备</p>
</blockquote>
<figure class="highlight html"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line"><span class="tag">&lt;<span class="name">div</span> <span class="attr">class</span>=<span class="string">"row"</span>&gt;</span></span><br><span class="line">	<span class="tag">&lt;<span class="name">div</span> <span class="attr">class</span>=<span class="string">"col-md-10 col-md-offset-1"</span>&gt;</span></span><br><span class="line">	...</span><br><span class="line">	<span class="tag">&lt;/<span class="name">div</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">div</span>&gt;</span></span><br></pre></td></tr></table></figure>
<h2 id="小结"><a href="#小结" class="headerlink" title="小结"></a>小结</h2><p>整个博文写下来感觉整个Demo也不并不难,23333333333333………….<br>但是每个人水平参差不齐<br>当你从第一步开始写时,你就明白到底难易了。。。。<br>反正我感觉有部分功能的实现过程很痛苦  ORZ</p>
<p><img src="https://s1.ax1x.com/2018/01/01/pSfme1.jpg" alt="ahaha"></p>

      
    </div>

    
      
      



      
      
    

    
      <footer class="post-footer">
        
          <div class="post-tags">
            
              <a href="/tags/开发/">开发</a>
            
              <a href="/tags/flask/">flask</a>
            
              <a href="/tags/python/">python</a>
            
              <a href="/tags/lxml/">lxml</a>
            
          </div>
        
        
        
  <nav class="post-nav">
    
      <a class="prev" href="/2017/10/27/各种注入姿势/">
        <i class="iconfont icon-left"></i>
        <span class="prev-text nav-default">各种注入姿势</span>
        <span class="prev-text nav-mobile">上一篇</span>
      </a>
    
    
      <a class="next" href="/2017/09/22/如何在flask实现form搜索/">
        <span class="next-text nav-default">如何在flask实现form搜索</span>
        <span class="prev-text nav-mobile">下一篇</span>
        <i class="iconfont icon-right"></i>
      </a>
    
  </nav>

      </footer>
    

  </article>


          </div>
          
  <div class="comments" id="comments">
      <div id="disqus_thread">
        <noscript>
          Please enable JavaScript to view the
          <a href="//disqus.com/?ref_noscript">comments powered by Disqus.</a>
        </noscript>
      </div> 
    </div>
  </div>


        </div>
      </main>

      <footer id="footer" class="footer">

  <div class="social-links">
    
      
        
          <a href="https://github.com/bay1" class="iconfont icon-github" title="github"></a>
        
      
    
      
        
          <a href="http://weibo.com/3190704711/profile?topnav=1&wvr=6&is_all=1" class="iconfont icon-weibo" title="weibo"></a>
        
      
    
      
    
      
    
      
    
    
    
  </div>


<div class="copyright">
  <span class="copyright-year">
    
    &copy; 
     
      2016 - 
    
    2018
    <span class="author">bay1</span>
  </span>
</div>
      </footer>

      <div class="back-to-top" id="back-to-top">
        <i class="iconfont icon-up"></i>
      </div>
    </div>

    
  
  <script type="text/javascript">
    var disqus_config = function () {
        this.page.url = 'https://bay1.top/2017/10/08/矿大新闻热点爬虫系统搬砖记录/';
        this.page.identifier = '2017/10/08/矿大新闻热点爬虫系统搬砖记录/';
        this.page.title = 'Python爬虫,收藏功能实现记录';
    };
    (function() {
    var d = document, s = d.createElement('script');

    s.src = '//https-blog-flywinky-top-1.disqus.com/embed.js';

    s.setAttribute('data-timestamp', +new Date());
    (d.head || d.body).appendChild(s);
    })();  
  </script>



    
  





  
    <script type="text/javascript" src="/lib/jquery/jquery-3.1.1.min.js"></script>
  

  
    <script type="text/javascript" src="/lib/slideout/slideout.js"></script>
  

  
    <script type="text/javascript" src="/lib/fancybox/jquery.fancybox.pack.js"></script>
  


    <script type="text/javascript" src="/js/src/even.js?v=2.6.0"></script>
<script type="text/javascript" src="/js/src/bootstrap.js?v=2.6.0"></script>
<script src="/js/prettify.js"></script>
<script type="text/javascript">
$(document).ready(function(){
 $('pre').addClass('prettyprint');
   prettyPrint();
 })
</script>
  </body>
</html>
